package mc.contents.etc;

import mc.contents.pojo.persistence.ShopInfo;
import org.apache.commons.lang.StringUtils;
import org.apache.http.*;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.params.ClientPNames;
import org.apache.http.client.params.CookiePolicy;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.scheme.SchemeSocketFactory;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.params.*;
import org.apache.http.util.EntityUtils;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.htmlcleaner.XPatherException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;

import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.*;
import java.util.*;

/**
 * Created with IntelliJ IDEA. <br/>
 * User: francis    <br/>
 * Date: 13-11-7    <br/>
 * Time: 10:23  <br/>
 */
@Component
public class CrawlerClient {

    private static Logger log = LoggerFactory.getLogger( CrawlerClient.class );
    public static final String DEFAULT_CHARSET = "utf8";
    private InnerHttpClient client;
    private Map<String, Integer> proxyList;

    public CrawlerClient() {
        //this.client = new InnerHttpClient();
    }

    public Map<String, Integer> getProxyList() {
        return proxyList;
    }

    public void setProxyList(Map<String, Integer> proxyList) {
        this.proxyList = proxyList;
    }

    private InnerHttpClient getInnerHttpClient() {
        if ( null == client ) {
            if ( null != proxyList && !proxyList.isEmpty() ) {
                client = new InnerHttpClient( proxyList );
            } else {
                client = new InnerHttpClient();
            }
        }
        return client;
    }

    public String crawlShopUrl( Long shopSid ) {
        try {
            String html = getInnerHttpClient().doQuery( "http://shop"+shopSid+".taobao.com/" );
            if( html!=null ){
                HtmlCleaner cleaner=new HtmlCleaner();
                TagNode node = cleaner.clean( new BufferedInputStream(
                        new ByteArrayInputStream( html.getBytes() ) ), DEFAULT_CHARSET );

                Object[] tagNodes =node.evaluateXPath("//span[@class='shop-rank']/a");//使用xpath方式提取标记

                for ( int indexIndexPage=0; indexIndexPage< tagNodes.length; indexIndexPage++ ) {
                    TagNode tagNode = (TagNode)tagNodes[indexIndexPage];
                    String href = tagNode.getAttributeByName("href");
                    if( StringUtils.isNotEmpty( href )){
                        return href.substring( href.lastIndexOf( "-" )+1,href.lastIndexOf( ".htm" ) );
                    }
                }
            }
        } catch ( XPatherException | IOException e ) {
            e.printStackTrace();
        }
        return null;
    }

    public void crawlShop4C( ShopInfo shopInfo ) {
        try{
            String html = getInnerHttpClient().doQuery( "http://rate.taobao.com/ShopService4C.htm?userNumId="+shopInfo.getShopUrl()+"&ua&shopID="+shopInfo.getSid()+"&isB2C="+("C".equalsIgnoreCase(shopInfo.getShopType()) ? "false" : "true" ));
            shopInfo.setShopService4c( html.trim() );
        }catch ( Exception e ){
            System.out.println("ip封锁了");
            e.printStackTrace();
        }
    }

    public static class InnerHttpClient {

        public static String[] USER_AGENTS = {
                "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:20.0) Gecko/20100101 Firefox/20.0",
                "Mozilla/5.0 (compatible; MSIE 9.0; qdesk 2.4.1263.203; Windows NT 6.1; WOW64; Trident/6.0)",
                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36",
                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20100101 Firefox/24.0",
                "User-Agent Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.71 (KHTML, like Gecko) Version/6.1 Safari/537.71",
                "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.69 Safari/537.17"
        };

        private static final Map<String, String> DEFAULT_HEAD_MAP = new HashMap<String, String>() {
            {
                put( "Pragma", "no-cache" );
                put( "Cache-Control", "max-age=0, no-cache, no-store, must-revalidate" );
                put( "Connection","keep-alive" );
                put( "Accept","text/html,application/xhtml+xml,application/xml,application/x-javascript;q=0.9,*/*;q=0.8" );
                put( "Accept-Language","zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3" );
                put( "Accept-Charset", "utf-8, gbk;q=0.7,*;q=0.7" );
            }
        };

        private static final String DEFAULT_CHARSET = "utf-8";

        private static final int DEFAULT_RETRY_TIMES = 3;

        private String charset;

        private int retryTimes;

        private List<ProxyConfig> proxies;
        private boolean proxyEnable = false;
        private Random random;

        public InnerHttpClient() {
            this( DEFAULT_CHARSET, DEFAULT_RETRY_TIMES );
        }

        InnerHttpClient(Map<String, Integer> proxyList) {
            this( DEFAULT_CHARSET, DEFAULT_RETRY_TIMES );
            if ( null != proxyList && !proxyList.isEmpty() ) {

                proxies = new ArrayList<>( proxyList.size() );
                for ( Map.Entry<String, Integer> e : proxyList.entrySet() ) {
                    String addr = e.getKey();
                    Integer port = e.getValue();
                    if ( null != addr && null != port ) {
                        proxies.add( new ProxyConfig( addr, port ) );
                    }
                }
                proxyEnable = true;
                random = new Random();
            }
        }

        InnerHttpClient(String charset, int retryTimes) {
            this.charset = charset;
            this.retryTimes = retryTimes;
        }

        String getCharset() {
            return charset;
        }

        ProxyConfig getProxyConfig() {
            int len = proxies.size();
            return proxies.get( random.nextInt( len ) );
        }

        void setCharset(String charset) {
            this.charset = charset;
        }

        public String doQuery(String url) throws IOException {
            return doQuery( url, 0 );
        }

        private String doQuery(String url, int queryTime) throws IOException {

            HttpGet get = new HttpGet( url );
            get = assemblyHeaders( get );

            String html = null;
            DefaultHttpClient client = getHttpClient();
            try {
                HttpResponse resp = client.execute( get );
                html = parseResp( url, resp, queryTime );
            } finally {
                client.getCookieStore().clear();
            }

            return html;
        }

        private String parseResp(String originalURL, HttpResponse resp, int queryTime) throws IOException {
            int status = resp.getStatusLine().getStatusCode();
            String html = "";

            if ( queryTime >= retryTimes ) {
                return html;
            }

            switch ( status ) {
                case HttpStatus.SC_OK: html = handleSuccessResp( resp ); break;
                case HttpStatus.SC_MOVED_PERMANENTLY:
                case HttpStatus.SC_MOVED_TEMPORARILY: html = handleMovedResp( resp ); break;
                default: html = doQuery( originalURL, ++queryTime );
            }

            return html;
        }

        private String handleSuccessResp( HttpResponse resp ) throws IOException {
            HttpEntity entity = resp.getEntity();
            String html = "";
            if ( null != entity ) {
                html = EntityUtils.toString( entity, charset );
                EntityUtils.consume(entity);
            }
            return html;
        }

        private String handleMovedResp( HttpResponse resp ) throws IOException {
            Header locationHeader = resp.getFirstHeader("Location");
            String location;
            String html = "";
            if (locationHeader != null) {
                location = locationHeader.getValue();
                //log.info("The page was redirected to:" + location);
                html = doQuery( location, 0 );
            }
//            else {
                //log.info("Location field value is null.");
//            }

            return html;
        }

        private HttpGet assemblyHeaders(HttpGet get) {
            for ( Map.Entry<String, String> e: DEFAULT_HEAD_MAP.entrySet() ) {
                get.setHeader( e.getKey(), e.getValue() );
            }

            Random r = new Random();
            String userAgent = USER_AGENTS[ r.nextInt( USER_AGENTS.length ) ];
            get.setHeader( "User-Agent", userAgent );

            return get;
        }

        private SchemeRegistry schemeRegistry = new SchemeRegistry();
        private PoolingClientConnectionManager cm;
        private boolean initialized = false;
        private void initHttpConfig() {
            schemeRegistry = new SchemeRegistry();
            if ( proxyEnable ) {
                schemeRegistry.register( new Scheme("http", 80, new SocksSchemeSocketFactory() ) );
            } else {
                schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
            }
            schemeRegistry.register( new Scheme("https", 443, SSLSocketFactory.getSocketFactory() ) );
            cm = new PoolingClientConnectionManager( schemeRegistry );
            cm.setMaxTotal( 10 );
            cm.setDefaultMaxPerRoute( 5 );
        }

        public DefaultHttpClient getHttpClient(){

            if ( !initialized ) {
                initHttpConfig();
                initialized = true;
            }

            HttpParams httpParams  = new BasicHttpParams();
            httpParams.setIntParameter( CoreConnectionPNames.CONNECTION_TIMEOUT, 8000 );
            httpParams.setIntParameter( CoreConnectionPNames.SO_TIMEOUT, 10000 );
            httpParams.setBooleanParameter( CoreProtocolPNames.USE_EXPECT_CONTINUE,  false );
            httpParams.setParameter( CoreProtocolPNames.PROTOCOL_VERSION, HttpVersion.HTTP_1_1 );
            httpParams.setParameter( ClientPNames.COOKIE_POLICY, CookiePolicy.IGNORE_COOKIES );

            if ( proxyEnable ) {
                ProxyConfig p = getProxyConfig();
                httpParams.setParameter( SocksSchemeSocketFactory.SOCKS_HOST, p.getAddress());
                httpParams.setParameter( SocksSchemeSocketFactory.SOCKS_PORT, p.getPort() );
            }

            return new DefaultHttpClient( cm, httpParams );
        }

        static class ProxyConfig {

            final String address;
            final Integer port;

            ProxyConfig(String address, Integer port) {
                this.address = address;
                this.port = port;
            }

            public String getAddress() {
                return address;
            }

            public Integer getPort() {
                return port;
            }

        }

        static class SocksSchemeSocketFactory implements SchemeSocketFactory {

            final static String SOCKS_HOST = "socks.host";
            final static String SOCKS_PORT = "socks.port";

            @Override
            public Socket connectSocket(final Socket socket, final InetSocketAddress remoteAddress,
                                    final InetSocketAddress localAddress, final HttpParams params) throws IOException,
                    UnknownHostException, ConnectTimeoutException {
                if (remoteAddress == null) {
                    throw new IllegalArgumentException("Remote address may not be null");
                }
                if (params == null) {
                    throw new IllegalArgumentException("HTTP parameters may not be null");
                }
                Socket sock;
                if (socket != null) {
                    sock = socket;
                } else {
                    sock = createSocket(params);
                }
                if (localAddress != null) {
                    sock.setReuseAddress(HttpConnectionParams.getSoReuseaddr(params));
                    sock.bind(localAddress);
                }
                int timeout = HttpConnectionParams.getConnectionTimeout(params);
                try {
                    sock.connect(remoteAddress, timeout);
                } catch (SocketTimeoutException ex) {
                    throw new ConnectTimeoutException("Connect to " + remoteAddress.getHostName() + "/" + remoteAddress.getAddress() + " time out");
                }
                return sock;
            }

            @Override
            public Socket createSocket(final HttpParams params) throws IOException {
                if (params == null) {
                    throw new IllegalArgumentException("Http parameters may not be null");
                }
                String proxyHost = (String) params.getParameter( SOCKS_HOST );
                Integer proxyPort = (Integer) params.getParameter( SOCKS_PORT );

                InetSocketAddress socksAddr = new InetSocketAddress(proxyHost, proxyPort);
                Proxy proxy = new Proxy( Proxy.Type.SOCKS, socksAddr );
                return new Socket(proxy);
            }

            @Override
            public boolean isSecure(final Socket sock) throws IllegalArgumentException {
                // TODO Auto-generated method stub
                return false;
            }

        }

    }

}
